# -*- coding: utf-8 -*-
from datetime import timedelta
from jms.dwd.tms.dwd_tmsnew_shipment_stop_union_base_dt import jms_dwd__dwd_tmsnew_shipment_stop_union_base_dt
from jms.dim.dim_sys_notice_base import jms_dim__dim_sys_notice_base
from jms.dwd.tab.dwd_tab_barscan_centersend_base_dt import jms_dwd__dwd_tab_barscan_centersend_base_dt
from jms.dwd.dwd_wide_abnormal_detail_waybill_dt import jms_dwd__dwd_wide_abnormal_detail_waybill_dt
from jms.dwd.tab.dwd_tab_reback_transfer_express_base import jms_dwd__dwd_tab_reback_transfer_express_base
from jms.dwd.tab.dwd_tab_barscan_loading_base_dt import jms_dwd__dwd_tab_barscan_loading_base_dt
from jms.dim.dim_network_whole_massage import jms_dim__dim_network_whole_massage
from jms.dwd.tab.dwd_tab_barscan_unloading_base_dt import jms_dwd__dwd_tab_barscan_unloading_base_dt
from jms.dwd.tab.dwd_tab_barscan_centerarrival_base_dt import jms_dwd__dwd_tab_barscan_centerarrival_base_dt
from jms.dwd.tab.dwd_tab_barscan_other_base_dt import jms_dwd__dwd_tab_barscan_other_base_dt
from jms.dwd.oms.dwd_yl_oms_oms_waybill_incre_dt import jms_dwd__dwd_yl_oms_oms_waybill_incre_dt
from jms.dim.dim_first_second_third_code_info import jms_dim__dim_first_second_third_code_info
from jms.dim.dim_tmsnew_vehicle_line import jms_dim__dim_tmsnew_vehicle_line
###from jms.ods.tms.yl_tms_vehicle_type import jms_ods__yl_tms_vehicle_type
from jms.dim.tms.dim_yl_tms_vehicle_type_base import jms_dim__dim_yl_tms_vehicle_type_base
from jms.dwd.tms.dwd_tmsnew_shipment_union_base_dt import jms_dwd__dwd_tmsnew_shipment_union_base_dt
from jms.dwd.tab.dwd_tab_barscan_sign_base_dt import jms_dwd__dwd_tab_barscan_sign_base_dt
from jms.dwd.tab.dwd_tab_barscan_collect_base_dt import jms_dwd__dwd_tab_barscan_collect_base_dt
from jms.time_sensor.time_after_05_30  import time_after_05_30


from utils.operators.cluster_for_spark_sql_operator import SparkSqlOperator

jms_dm__dm_cn_departure_transfer_timely_dt = SparkSqlOperator(
    task_id='jms_dm__dm_cn_departure_transfer_timely_dt',
    email=['zhangqinglin@jtexpress.com', 'yl_bigdata@yl-scm.com'],
    name='jms_dm__dm_cn_departure_transfer_timely_dt_{{ execution_date | date_add(1) | cst_ds }}',
    email_on_retry=True,
    retries=1,
    sql='jms/dm/dm_cn_departure_transfer_timely_dt/execute.sql',
    yarn_queue='pro',
    driver_memory='32G',
    driver_cores=8,
    executor_memory='12G',
    executor_cores=4,
    pool_slots=5,
    pool='unlimited_pool',
    num_executors=100,  # spark.dynamicAllocation.enabled 为 True 时，num_executors 表示最少 Executor 数
    conf={'spark.executor.memoryOverhead': '5G',
          'spark.dynamicAllocation.enabled': 'true',
          'spark.shuffle.service.enabled': 'true',
          'spark.dynamicAllocation.maxExecutors': 100,
          'spark.sql.sources.partitionOverwriteMode': 'dynamic',
          'spark.dynamicallocation.enabled': 'true',
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 120,
          'spark.sql.shuffle.partitions': 1000,
          'spark.shuffle.consolidateFiles': 'true',
          'spark.shuffle.memoryFraction': '0.8',
          'spark.shuffle.file.buffer':'64k',
          'spark.executor.extraJavaOptions': '-XX:+UseG1GC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=2'
          },
    execution_timeout=timedelta(hours=1),
    priority_weight=20,
)

jms_dm__dm_cn_departure_transfer_timely_dt << [
    jms_dwd__dwd_tmsnew_shipment_stop_union_base_dt,
    jms_dim__dim_sys_notice_base,
    jms_dwd__dwd_tab_barscan_centersend_base_dt,
    jms_dwd__dwd_wide_abnormal_detail_waybill_dt,
    jms_dwd__dwd_tab_reback_transfer_express_base,
    jms_dwd__dwd_tab_barscan_loading_base_dt,
    jms_dim__dim_network_whole_massage,
    jms_dwd__dwd_tab_barscan_unloading_base_dt,
    jms_dwd__dwd_tab_barscan_centerarrival_base_dt,
    jms_dwd__dwd_tab_barscan_other_base_dt,
    jms_dwd__dwd_yl_oms_oms_waybill_incre_dt,
    jms_dim__dim_first_second_third_code_info,
    jms_dim__dim_tmsnew_vehicle_line,
    jms_dim__dim_yl_tms_vehicle_type_base,
    jms_dwd__dwd_tmsnew_shipment_union_base_dt,
    jms_dwd__dwd_tab_barscan_sign_base_dt,
    jms_dwd__dwd_tab_barscan_collect_base_dt,
    time_after_05_30,
]
